# -*- coding: utf-8 -*-
# @Time    : 2020/1/9 17:27
# @Author  : Damn7Kx
# @Software: PyCharm
import datetime
import json
import random
import time
import uuid
from urllib.parse import quote, urlencode
from w3lib.html import remove_tags
import scrapy
from NewsSpider.items import NewsItem
from NewsSpider.tools.utils import Utils
from NewsSpider.tools.filter_time import Times
from NewsSpider.tools.redis_db import Redis_DB


class ChuDianNews(scrapy.Spider):
    '''
        触电新闻APP参数加密 X-ITOUCHTV-Ca-Signature hmac256
        盐值 qmiHeB9bKgowHqxRv0prc2cPN2EwXL1HOYu3DPiYCcaYxyxdFIyT5mAfBmr0UKPO
       触电新闻 get请求 固定api获取 pageNum翻页  channelId 频道 Timestamp时间戳 ts 时间戳
    '''

    name = 'Chudian'
    t = Times()
    redis = Redis_DB()
    types = [str(i) for i in range(5, 20)]
    types2 = ['544', '533', '539', '29', '545', '540', '538', '556',
              '583', '559', '582', '569', '-1']
    type3 = types + types2

    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'NewsSpider.middlewares.ProxyIPMiddleware': 544,
        },
        'ITEM_PIPELINES': {
            'NewsSpider.pipelines.KafkaPipeline': 544,
        }
    }
    headers = {
        'X-ITOUCHTV-Ca-Key': '04039368653554864194910691389924',
        'X-ITOUCHTV-CONN-TYPE': 'WIFI',
        'X-ITOUCHTV-DEVICE-SERIAL-NUM': 'G011A',
        'X-ITOUCHTV-CLIENT': 'NEWS_APP',
        'X-ITOUCHTV-TERM': 'G011A',
        'X-ITOUCHTV-INSTALL-CHANNEL': 's360',
        'X-ITOUCHTV-RESOLUTION': '1280,720',
        'X-ITOUCHTV-DEVICE-SERIAL-NUM2': '47952412',
        'X-ITOUCHTV-MEID': '',
        'X-ITOUCHTV-WEBVIEW-UA': 'Mozilla/5.0 (Linux; Android 7.1.2; G011A Build/N2G48H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.100 Safari/537.36',
        'X-ITOUCHTV-DEVICE-IMSI': '460022412271245',
        'X-ITOUCHTV-CCODE': '3701',
        'X-ITOUCHTV-DEVICE-ID': 'IMEI_869342479554350',
        'X-ITOUCHTV-ICCID': '89860011749638479373',
        'X-ITOUCHTV-CARRIER-CODE': '46000',
        'X-ITOUCHTV-MAC': 'A2:13:1D:C2:3B:9D',
        'X-ITOUCHTV-IDFA': '',
        'X-ITOUCHTV-APP-VERSION': '2.3.7',
        'X-ITOUCHTV-Ca-Signature': 'BbtKmIWHbOXNopTaI4dv5Vb8tSdzsQl0+4YXIdb6X5A=',
        'X-ITOUCHTV-Ca-Timestamp': '1578563003077',
        'X-ITOUCHTV-PCODE': '37',
        'X-ITOUCHTV-TS': '1578466346461',
        'X-ITOUCHTV-OSVS': '7.1.2',
        'X-ITOUCHTV-ACODE': '0',
        'X-ITOUCHTV-ANDROIDID': 'c2a5f2b10fd596e8',
        'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 7.1.2; G011A Build/N2G48H)',
        'Connection': 'Keep-Alive',
        'Accept-Encoding': 'gzip'
    }

    key = 'qmiHeB9bKgowHqxRv0prc2cPN2EwXL1HOYu3DPiYCcaYxyxdFIyT5mAfBmr0UKPO'

    def start_requests(self):
        for t in self.type3:
            timestamp = str(int(time.time() * 1000))
            url = f"https://api.itouchtv.cn/newsservice/v3/uniteChannelNews?installChannel=s360&longitude=0.0&channelType=0&refreshType=0&devicePlatform=android&romVersion=google&snapshotNumber=0&oldestTime=0&deviceBrand=google&channelId={t}&md5Hash=&focusPictureHash=&pCode=37&cCode=3701&aCode=0&osVersion=7.1.2&pageNum=1&latitude=0.0&deviceModel=G011A&language=zh&ac=1"
            self.headers['X-ITOUCHTV-Ca-Timestamp'] = timestamp
            self.headers['X-ITOUCHTV-TS'] = timestamp
            plaintText = "GET\n" + url + "\n" + timestamp+ "\n"
            self.headers['X-ITOUCHTV-Ca-Signature'] = Utils.hmac_sha256(plaintText,self.key)
            yield scrapy.Request(url, callback=self.parse_text, headers=self.headers, dont_filter=True,
                                 meta={"type": t, "pageNum": 1})

    def parse_text(self, response):
        print("正在访问列表页:", response.url)
        t = response.meta['type']
        number = response.meta['pageNum']
        datas = json.loads(response.text)
        data_ = datas['items']
        if number > 7:
            pass
        else:
            timestamp = str(int(time.time() * 1000))
            # pageNum设置为0
            url = f"https://api.itouchtv.cn/newsservice/v3/uniteChannelNews?installChannel=s360&longitude=0.0&channelType=0&refreshType=0&devicePlatform=android&romVersion=google&snapshotNumber=0&oldestTime=0&deviceBrand=google&channelId={t}&md5Hash=&focusPictureHash=&pCode=37&cCode=3701&aCode=0&osVersion=7.1.2&pageNum=0&latitude=0.0&deviceModel=G011A&language=zh&ac=1"
            self.headers['X-ITOUCHTV-Ca-Timestamp'] = timestamp
            self.headers['X-ITOUCHTV-TS'] = timestamp
            plaintText = "GET\n" + url + "\n" + timestamp + "\n"
            self.headers['X-ITOUCHTV-Ca-Signature'] = Utils.hmac_sha256(plaintText, self.key)
            yield scrapy.Request(url, callback=self.parse_text, headers=self.headers, dont_filter=True,
                                 meta={"type": t, "pageNum":number + 1})
        if data_:
            for d in data_:
                data_2 = json.loads(d['data'])
                try:
                    sid = data_2['sid']
                    content_url = f'https://api.itouchtv.cn/newsservice/v12/newscontent?sid={sid}'
                    timestamp = str(int(time.time() * 1000))
                    self.headers['X-ITOUCHTV-Ca-Timestamp'] = timestamp
                    self.headers['X-ITOUCHTV-TS'] = timestamp
                    plaintText = "GET\n" + content_url + "\n" + timestamp + "\n"
                    self.headers['X-ITOUCHTV-Ca-Signature'] = Utils.hmac_sha256(plaintText, self.key)
                    yield scrapy.Request(content_url, headers=self.headers, dont_filter=True, callback=self.parse)
                except:
                    try:
                        list2 = data_2['contentItemList']
                        for i2 in list2:
                            datas = json.loads(i2['data'])
                            sid = datas['sid']
                            content_url = f'https://api.itouchtv.cn/newsservice/v12/newscontent?sid={sid}'
                            timestamp = str(int(time.time() * 1000))
                            self.headers['X-ITOUCHTV-Ca-Timestamp'] = timestamp
                            self.headers['X-ITOUCHTV-TS'] = timestamp
                            plaintText = "GET\n" + content_url + "\n" + timestamp + "\n"
                            self.headers['X-ITOUCHTV-Ca-Signature'] = Utils.hmac_sha256(plaintText, self.key)
                            yield scrapy.Request(content_url, headers=self.headers, dont_filter=True,
                                                 callback=self.parse)
                    except:
                        pass


    def parse(self, response):
        if response.text is not None:
            item = NewsItem()
            datas = json.loads(response.text)
            url = datas['thisNews']['fromUrl']
            if url is None:
                # print("url为空")
                pass
            else:
                id = Utils.url_hash(url)
                item['url'] = url
                item['id'] = id
                if self.redis.check_exist_2("wenzhangquchong", id, '') == 0:
                    print('该id:%s已存在' % id)
                else:
                    item['title'] = datas['thisNews']['title']
                    pubdate_datetime= datas['thisNews']['updateTime']
                    pubdate_time = Utils.process_timestamp(pubdate_datetime)
                    pubdate = str(self.t.datetimes(pubdate_time))
                    item['pubdate'] = pubdate
                    if not self.t.time_is_Recent(pubdate):
                        pass
                    else:
                        try:
                            html = datas['content']
                        except:
                            html = ''
                        item['content'] = remove_tags(html)
                        try:
                            item['author'] = datas['thisNews']['weMediaName']
                        except:
                            item['author'] = ''
                        item['formats'] = "app"
                        item['dataSource'] = datas['thisNews']['weMediaName']
                        item['serchEnType'] = "触电新闻"
                        try:
                            item['html'] = html
                        except:
                            item['html'] = ''
                        item['updateTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                        item['collectProcess'] = 'crawl_news'
                        item['serverIp'] = "113.128.12.74"
                        yield item
